***This code replicate Table 2 Performance evaluation of different generative AI models

cd ""

**Human annotation of 200 examples 
use "Table2.dta",clear
global v1 raw_type_gpt4omini-cons_googlegeminiflash15
sum $v1


*0 means true positive cases; missing value is postive cases
foreach v of varlist $v1 {
    replace `v'=0 if `v'==.
}

*calculate performance matrix
foreach v of varlist $v1 {
  qui sum `v' if `v'==0 /*true positve*/
   scalar TP=r(N)
  qui sum `v' if `v'==1 /*false positve*/
   scalar FP=r(N)  
 qui sum `v' if `v'==2 /*true negative*/
   scalar TN=r(N)
 qui sum `v' if `v'==3 /*false negative*/
   scalar FN=r(N)
   
scalar Accuracy=(TP+TN)/200
scalar Precision=(TP)/(TP+FP)
scalar Recall=(TP)/(TP+FN)
scalar F1=2*Precision*Recall/(Precision+Recall)   

matrix P_`v'=[Accuracy,Precision,Recall,F1]
 di "`v': " `v'
matrix list P_`v'  
}


*Accuracy table

scalar x0 = P_raw_type_gpt4omini[1,1]
scalar x1 = P_normed_tone_gpt4omini[1,1]
scalar x2 = P_cause_gpt4omini[1,1]
scalar x3 = P_cons_gpt4omini[1,1]
matrix Accuracy_GPT = (x0, x2, x3, x1)

scalar x0 = P_raw_type_anthropicclaude35ha[1,1]
scalar x1 = P_normed_tone_anthropicclaude35[1,1]
scalar x2 = P_cause_anthropicclaude35haiku[1,1]
scalar x3 = P_cons_anthropicclaude35haiku[1,1]
matrix Accuracy_Haiku = (x0, x2, x3, x1)

scalar x0 = P_raw_type_googlegeminiflash15[1,1]
scalar x1 = P_normed_tone_googlegeminiflash[1,1]
scalar x2 = P_cause_googlegeminiflash15[1,1]
scalar x3 = P_cons_googlegeminiflash15[1,1]
matrix Accuracy_Flash= (x0, x2, x3, x1)

matrix Accuracy=[Accuracy_GPT \ Accuracy_Haiku \ Accuracy_Flash]
matrix list Accuracy


*Accuracy table

scalar x0 = P_raw_type_gpt4omini[1,1]
scalar x1 = P_normed_tone_gpt4omini[1,1]
scalar x2 = P_cause_gpt4omini[1,1]
scalar x3 = P_cons_gpt4omini[1,1]
matrix Accuracy_GPT = (x0, x2, x3, x1)

scalar x0 = P_raw_type_anthropicclaude35ha[1,1]
scalar x1 = P_normed_tone_anthropicclaude35[1,1]
scalar x2 = P_cause_anthropicclaude35haiku[1,1]
scalar x3 = P_cons_anthropicclaude35haiku[1,1]
matrix Accuracy_Haiku = (x0, x2, x3, x1)

scalar x0 = P_raw_type_googlegeminiflash15[1,1]
scalar x1 = P_normed_tone_googlegeminiflash[1,1]
scalar x2 = P_cause_googlegeminiflash15[1,1]
scalar x3 = P_cons_googlegeminiflash15[1,1]
matrix Accuracy_Flash= (x0, x2, x3, x1)

matrix Accuracy=[Accuracy_GPT \ Accuracy_Haiku \ Accuracy_Flash]
matrix rownames Accuracy = GPT Haiku Flash
matrix colnames Accuracy = Type Cause Effect Tone
matrix list Accuracy

*Precision table

scalar x0 = P_raw_type_gpt4omini[1,2]
scalar x1 = P_normed_tone_gpt4omini[1,2]
scalar x2 = P_cause_gpt4omini[1,2]
scalar x3 = P_cons_gpt4omini[1,2]
matrix Precision_GPT = (x0, x2, x3, x1)

scalar x0 = P_raw_type_anthropicclaude35ha[1,2]
scalar x1 = P_normed_tone_anthropicclaude35[1,2]
scalar x2 = P_cause_anthropicclaude35haiku[1,2]
scalar x3 = P_cons_anthropicclaude35haiku[1,2]
matrix Precision_Haiku = (x0, x2, x3, x1)

scalar x0 = P_raw_type_googlegeminiflash15[1,2]
scalar x1 = P_normed_tone_googlegeminiflash[1,2]
scalar x2 = P_cause_googlegeminiflash15[1,2]
scalar x3 = P_cons_googlegeminiflash15[1,2]
matrix Precision_Flash= (x0, x2, x3, x1)

matrix Precision=[Precision_GPT \ Precision_Haiku \ Precision_Flash]
matrix rownames Precision = GPT Haiku Flash
matrix colnames Precision = Type Cause Effect Tone

matrix list Precision

*Recall table

scalar x0 = P_raw_type_gpt4omini[1,3]
scalar x1 = P_normed_tone_gpt4omini[1,3]
scalar x2 = P_cause_gpt4omini[1,3]
scalar x3 = P_cons_gpt4omini[1,3]
matrix Recall_GPT = (x0, x2, x3, x1)

scalar x0 = P_raw_type_anthropicclaude35ha[1,3]
scalar x1 = P_normed_tone_anthropicclaude35[1,3]
scalar x2 = P_cause_anthropicclaude35haiku[1,3]
scalar x3 = P_cons_anthropicclaude35haiku[1,3]
matrix Recall_Haiku = (x0, x2, x3, x1)

scalar x0 = P_raw_type_googlegeminiflash15[1,3]
scalar x1 = P_normed_tone_googlegeminiflash[1,3]
scalar x2 = P_cause_googlegeminiflash15[1,3]
scalar x3 = P_cons_googlegeminiflash15[1,3]
matrix Recall_Flash= (x0, x2, x3, x1)

matrix Recall=[Recall_GPT \ Recall_Haiku \ Recall_Flash]
matrix rownames Recall = GPT Haiku Flash
matrix colnames Recall = Type Cause Effect Tone
matrix list Recall

*F1 table

scalar x0 = P_raw_type_gpt4omini[1,4]
scalar x1 = P_normed_tone_gpt4omini[1,4]
scalar x2 = P_cause_gpt4omini[1,4]
scalar x3 = P_cons_gpt4omini[1,4]
matrix F1_GPT = (x0, x2, x3, x1)

scalar x0 = P_raw_type_anthropicclaude35ha[1,4]
scalar x1 = P_normed_tone_anthropicclaude35[1,4]
scalar x2 = P_cause_anthropicclaude35haiku[1,4]
scalar x3 = P_cons_anthropicclaude35haiku[1,4]
matrix F1_Haiku = (x0, x2, x3, x1)

scalar x0 = P_raw_type_googlegeminiflash15[1,4]
scalar x1 = P_normed_tone_googlegeminiflash[1,4]
scalar x2 = P_cause_googlegeminiflash15[1,4]
scalar x3 = P_cons_googlegeminiflash15[1,4]
matrix F1_Flash= (x0, x2, x3, x1)

matrix F1=[F1_GPT \ F1_Haiku \ F1_Flash]
matrix rownames F1 = GPT Haiku Flash
matrix colnames F1 = Type Cause Effect Tone
matrix list F1


//  Excel export
putexcel set Table2.xlsx, replace

// Accuracy matrix (goes to A1)
putexcel B1=matrix(Accuracy), names
putexcel A2 = "Accuracy"

// Precision matrix (goes below Accuracy, e.g. at row 6)
putexcel B6=matrix(Precision), names
putexcel A7 = "Precision"

// Recall matrix (goes below Precision, e.g. at row 11)
putexcel B11=matrix(Recall), names
putexcel A12 = "Recall"

// F1 matrix (goes below Precision, e.g. at row 16)
putexcel B16=matrix(F1), names
putexcel A17 = "F1"
